
#include "stdafx.h"
//#include "ds_tokenizer.h"

#define toSTRID(X) (X)

namespace ds {

const dsSTRING CPP_SYMBOLS = "+-*/%|&^~!()[]{}<>=,.;?:";

/*************************************************************************/
/* ds::TokenizeCppLike 
/*
/*************************************************************************/
void TokenizeCppLike(const dsSTRING & input, dsVECTOR<TOKEN> & tokens)
{
   enum {
      ST_DEFAULT,
      ST_WORD,
      // TODO: support literals
      // TODO: support comments (line & block)
   } state = ST_DEFAULT;

   int line = 1;
   int pos  = 0;
   dsSTRING value;
   TOKEN token;

   for (int i = 0; i < input.Length(); ++i) {
      const char cur = input[i];
      const char la1 = (i + 1 < input.Length()) ? input[i + 1] : 0;

      ++pos;

      switch (state) {
      case ST_DEFAULT:
         // skip spaces
         if (cur <= ' ') {
            if (line == '\n') {
               ++line;
               pos = 0;
            }
            continue;
         }

         token.line  = line;

         value.Clear();
         value += cur;

         if (IsSymbol(cur)) {
            token.token  = TOKEN_USER;
            token.value = toSTRID(value);
            tokens     += token;
            break;
         }

         if (IsWordLetter(cur, true)) {
            token.token  = TOKEN_WORD;

            if (IsWordLetter(la1)) { // check for single letter word
               state       = ST_WORD;
               break;
            }

            token.value = toSTRID(value);
            tokens += token;
         }
         break;

      case ST_WORD:
         STRONG_ASSERT(IsWordLetter(cur));
         value += cur;

         if (!IsWordLetter(la1)) {
            token.value = toSTRID(value);
            tokens += token;
            state = ST_DEFAULT;
         }
         break;

      default:
         STRONG_ASSERT(!"Unknown state");
         break;
      }
   }
}

/******************************************************************************/
/* ds::CPP_LIKE_TOKENIZER::RegisterToken
/*
/******************************************************************************/
void CPP_LIKE_TOKENIZER::RegisterToken(int type, char symbol)
{
   STRONG_ASSERT(type >= TOKEN_USER);

   USER_TOKEN token;
   token.token    = type;
   token.pattern  = symbol;

   tokens.PushBack(token);
}

/******************************************************************************/
/* ds::CPP_LIKE_TOKENIZER::RegisterCppTokens
/*
/******************************************************************************/
void CPP_LIKE_TOKENIZER::RegisterCppSymbols(int type)
{
   for (int i = 0; i < CPP_SYMBOLS.Length(); ++i) {
      RegisterToken(type, CPP_SYMBOLS[i]);
   }
}

/******************************************************************************/
/* ds::CPP_LIKE_TOKENIZER::DisableToken
/*
/******************************************************************************/
void CPP_LIKE_TOKENIZER::DisableToken(int type, bool disable)
{
   STRONG_ASSERT(type >= 0 && type < TOKEN_USER);
   _SetState(disabledTokens, (1 << type), disable);
}

/******************************************************************************/
/* ds::CPP_LIKE_TOKENIZER::ParseComments
/*
/******************************************************************************/
bool CPP_LIKE_TOKENIZER::ParseComments(dsSTRING text, int & index, int & line, TOKENIZER_ERROR_CALLBACK * err)
{
   if (text[index] != '/') {
      return true;
   }

   // line comment
   if (text[index + 1] == '/') {
      while (text[index] != '\n') {
         ++index;
      }

      ++line;
      return true;
   }

   // block comment
   if (text[index + 1] == '*') {
      ++index;

      int lll = line; // store /* line

      while (index < text.Length()) {
         if (text[index] == '*' && text[index + 1] == '/') {
            ++index;
            return true;
         }

         if (text[index] == '\n') {
            ++line;
         }
         ++index;
      }

      if (err != NULL) {
         err->Error(TOKEN_ERR_EOF_IN_COMMENT, lll);
      } else {
         ASSERT(!"End of file in block comment");
      }
      return false;
   }
 
   return true;
}

/******************************************************************************/
/* ds::CPP_LIKE_TOKENIZER::Tokenize
/*
/******************************************************************************/
bool CPP_LIKE_TOKENIZER::Tokenize(dsSTRING text, dsVECTOR<TOKEN> & res, TOKENIZER_ERROR_CALLBACK * err /* = NULL */)
{
   if (text.IsEmpty()) {
      return true;
   }

   enum {
      ST_DEFAULT,
      ST_WORD,
      ST_INT,
      ST_FLOAT,
      ST_STRING,
   } state = ST_DEFAULT;

   int line = 1;
   TOKEN token;
   dsSTRING value;
   
   ds::Sort(tokens);

   if (text[text.Length()] != '\n') {
      text += "\n"; // make sure the last symbol is something useless
   }

   for (int index = 0; index < text.Length(); ++index) {
      char ch = text[index];

      switch (state) {
         case ST_DEFAULT:
            {
               if (IsSpace(ch)) {
                  if (ch == '\n') {
                     ++line;
                  }
                  continue;
               }

               token.line = line;

               // identifier
               if (IsWordLetter(ch, true)) {
                  token.token = TOKEN_WORD;
                  value = ch;

                  state = ST_WORD;
                  continue;
               }

               // number starting with sign
               bool isNumber = false;
               if (ch == '+' || ch == '-') {
                  int lll = line;

                  // make sure it's a beginning of a number
                  for (int k = index + 1; k < text.Length(); ++k) {
                     if (IsSpace(text[k])) {
                        if (text[k] == '\n') {
                           ++lll;
                        }
                        continue;
                     }

                     // skip comments
                     if (text[k] == '/') {
                        if (text[k + 1] == '/' || text[k + 1] == '*') {
                           if (!ParseComments(text, k, lll, err)) {
                              return false;
                           }

                           continue;
                        }
                     }

                     if (IsDigit(text[k])) {
                        isNumber = true;
                        index = k - 1;
                        line  = lll;
                     }
                     break;
                  }
               }

               // number
               if (isNumber || IsDigit(ch)) {
                  if (ch != '+') {
                     value = ch;
                  } else {
                     value.Clear();
                  }

                  token.token = TOKEN_INT;

                  state = ST_INT;
                  continue;
               }

               // string
               if (ch == '"') {
                  token.token = TOKEN_STRING;
                  value.Clear();
                  state = ST_STRING;
                  continue;
               }

               // comments
               if (ch == '/') {
                  if (text[index + 1] == '/' || text[index + 1] == '*') {
                     if (!ParseComments(text, index, line, err)) {
                        return false;
                     }
                     continue;
                  }
               }

               // check for user tokens
               value = ch;
               bool found = false;
               for (int k = 0; k < tokens.Length(); ++k) {
                  STRONG_ASSERT(tokens[k].pattern.Length() == 1);
                  if (ch == tokens[k].pattern[0]) {
                     token.token = tokens[k].token;
                     token.value = toSTRID(value);
                     res += token;
                     
                     found = true;
                     break;
                  }
               }
               if (found) {
                  continue;
               }

               if (err != NULL) {
                  err->Error(TOKEN_ERR_UNEXPECTED_SYMBOL, line, ch);
               } else {
                  ASSERT(!"Unexpected symbol");
               }

               return false;
            }
            break;

         case ST_WORD:
            if (IsWordLetter(ch)) {
               value += ch;
               continue;
            }

            token.value = toSTRID(value);
            res += token;

            state = ST_DEFAULT;
            --index;
            break;

         case ST_INT:
         case ST_FLOAT:
            if (IsDigit(ch)) {
               value += ch;
               continue;
            }
            if (state == ST_INT && ch == '.') {
               token.token = TOKEN_FLOAT;
               value += ch;
               state = ST_FLOAT;
               continue;
            }

            token.value = toSTRID(value);
            res += token;
            --index;
            state = ST_DEFAULT;
            break;

         case ST_STRING:
            if (ch == '\\' && text[index + 1] != '\n') {
               char ch2 = text[index + 1];
               switch (ch2) {
                  case 'n': value += '\n'; break;
                  case 'r': value += '\r'; break;
                  case 't': value += '\t'; break;
                  case '\\': value += '\\'; break;
                  default: value += ch2; break;
               }

               ++index;
               continue;
            }
            if (ch == '\n') {
               if (err != NULL) {
                  err->Error(TOKEN_ERR_EOLN_IN_LITERAL, line);
               } else {
                  ASSERT(!"New line in string constant");
               }
               return false;
            }
            if (ch != '"') {
               value += ch;
               continue;
            }

            token.value = toSTRID(value);
            res += token;
            state = ST_DEFAULT;
            break;
      }
   }

   return true;
}

/******************************************************************************/
/* ds::GetIndentation
/*
/******************************************************************************/
int GetIndent(const dsSTRING & text) {
   dsVECTOR<dsSTRING> tmp;
   return GetIndent(text, tmp);
}

/******************************************************************************/
/* ds::GetIndentation
/*
/******************************************************************************/
int GetIndent(const dsSTRING & text, dsVECTOR<dsSTRING> & lines)
{
   lines.Resize(1);
   lines[0].Clear();
   for (int i = 0; i < text.Length(); ++i) {
      if (text[i] == '\n') {
         lines.PushBack();
         continue;
      }

      lines.Back() += text[i];
   }

   int minIndent = text.Length();

   for (int i = 0; i < lines.Length(); ++i) {
      dsSTRING & line = lines[i];
      line.TrimRight();

      if (line.IsEmpty()) {
         continue;
      }

      int indent = 0;
      for (int k = 0; k < line.Length(); ++k) {
         if (ds::IsSpace(line[k])) {
            ++indent;
         } else {
            break;
         }
      }

      m3dUpdateMin(minIndent, indent);
   }

   return minIndent;
}

/******************************************************************************/
/* ds::BuildStringFromLines
/*
/******************************************************************************/
dsSTRING BuildStringFromLines(const dsVECTOR<dsSTRING> & lines, int start /*= 0*/, int count /*= -1*/)
{
   if (count == -1) {
      count = lines.Length() - start;
   }

   dsSTRING res;
   for (int i = 0; i < count; ++i) {
      res += lines[i + start];
      if (i + 1 != count) {
         res += "\n";
      }
   }

   return res;
}

/******************************************************************************/
/* ds::SetIndent
/*
/******************************************************************************/
dsSTRING SetIndent(const dsSTRING & text, int indent)
{
   dsVECTOR<dsSTRING> lines;
   int prevIndent = GetIndent(text, lines);

   dsSTRING newIndent(' ', indent);

   dsSTRING res;
   for (int i = 0; i < lines.Length(); ++i) {
      dsSTRING & line = lines[i];

      if (!line.IsEmpty()) {
         res += newIndent;
         res += line.SubStr(prevIndent, line.Length() - prevIndent);
      }

      res += "\n";
   }

   return res;
}

};

